package jude.java.money.article;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import jude.java.money.net.HttpCampaign;
import jude.java.money.tools.Tidy;
import jude.java.util.Utils;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.lang.StringUtils;

public class CommonArticleGather extends HttpCampaign implements Runnable {
//	private static final String SAVE_PATH = "E:\\articles\\";
	private static final String SAVE_PATH = Gathers.ARTICLE_LIBRARY_PATH;
	private static final boolean SAVE_TITLE = false;
	
	private static int siteId;
	
	private final ArticleSite site;
	private HttpClient client;
	private boolean shield;
	
	private int max;		//最大采集数(含已经存在的)
	private int count1;		//采集总数
	private int count2;		//采集总数, 含已经存在的
	
	//用于防止一直重复读取同一页面(页码变化, 但内容可能是一样的)
	private StringBuffer previous;		//记录上次读取内容摘要
	private StringBuffer current;		//当前读取内容摘要
	
	public CommonArticleGather(ArticleSite site) {
		this.site = site;
		previous = new StringBuffer();
		current = new StringBuffer();
	}
	
	public void run() {
		client = getHttpClient();
		Page page = null;
		String ori = null;
		if (site.queryable()) {
			page = site.getQueryPage();
			ori = getQueryUrl(page, site.getQueryKey(), site.getQuery());
		} else {
			page = site.getListPage();
			if (page != null)
				ori = page.getUrl();
		}
		if (page == null) {
			handleNoListPages();
			return;
		}
		if (site.getPageCount() > 0) {
			previous.setLength(0);
			current.setLength(0);
			for (int i = 0; i < site.getPageCount(); i++) {
				int pageIndex = site.getStartPage() + i;
				String url = getPageUrl(ori, pageIndex);
				System.out.println(url);
				GetMethod get = getGetMethod(url, ori, page.isFollowRedirects());
				try {
					String resp = execute(client, get);
					if (checkShield(resp))
						break;
					sleep();
					if (!handlerListResponse(page, resp))
						continue;
				} catch (IOException e) {
					e.printStackTrace();
				} finally {
					get.releaseConnection();
				}
				if (max > 0 && count2 >= max)
					break;
				if (current.length() > 0 && current.toString().equals(previous.toString())) {
					System.err.println("repeat");
					break;
				}
//				System.out.println(previous);
//				System.out.println(current);
				previous.setLength(0);
				previous.append(current);
				current.setLength(0);
			}
		} else {
			GetMethod get = getGetMethod(ori, ori, page.isFollowRedirects());
			try {
				String resp = execute(client, get);
				if (checkShield(resp))
					return;
				sleep();
				handlerListResponse(page, resp);
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				get.releaseConnection();
			}
		}
	}
	
	protected String execute(HttpClient client, HttpMethodBase method) throws IOException {
		int statusCode = client.executeMethod(method);
		if (statusCode != HttpStatus.SC_OK && !(site.isHttp403() && statusCode == HttpStatus.SC_FORBIDDEN)) {
			System.err.println(statusCode);
			method.releaseConnection();
			return null;
		}
		String resp = getResponse(method, -1);
		method.releaseConnection();
		return resp;
	}
	
	public CommonArticleGather setMax(int max) {
		this.max = max;
		return this;
	}
	
	private String getQueryUrl(Page page, String queryKey, String query) {
		Map<String, String> params = url2map(page.getUrl());
		try {
			params.put(queryKey, URLEncoder.encode(query, REQUEST_ENCODING));
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return map2url(params);
	}
	
	private boolean handlerListResponse(Page page, String listContent) {
		if (!successful(page, listContent)) {
			System.out.println("失败页面");
			return false;
		}
		Pattern p = Pattern.compile(page.getContentRegex());
		Matcher m = p.matcher(listContent);
		int n = 0;
		int[] groups = page.getGroups();
		if (groups == null || groups.length < 3)
			groups = new int[] {1, 2, 3};
		int found = 0;
		while (m.find(n)) {
			//url, id, title
			found ++;
			String u = m.group(groups[0]);	//URLDecoder.decode(m.group(groups[0]), REQUEST_ENCODING);
			String id = m.group(groups[1]);
			String title = m.group(groups[2]);
			u = StringUtils.replace(u, "&amp;", "&");
			if (!handleContentPage(page.getUrl(), u, id, title))
				return false;
			n = m.end(0);
			current.append(id).append("/");
			if (count2 % 10 == 0)
				System.out.println(count1 + "/" + count2 + "/" + max);
			if (max > 0 && count2 >= max)
				return false;
		}
		System.out.println("找到 " + found + " 篇");
		if (found == 0) {
			System.out.println(page.getContentRegex());
			System.out.println(listContent);
			return false;
		}
		return true;
	}
	
	private boolean successful(Page page, String content) {
		if (content == null)
			return false;
		String[] fails = page.getFailures();
		if (fails != null)
			for (int i = 0; i < fails.length; i++) {
				if (StringUtils.isEmpty(fails[i]))
					continue;
				if (content.contains(fails[i]))
					return false;
			}
		String[] succs = page.getSuccesss();
		if (succs != null)
			for (int i = 0; i < succs.length; i++) {
				if (StringUtils.isEmpty(succs[i]))
					continue;
				if (!content.contains(succs[i]))
					return false;
			}
		return true;
	}
	
	/** 无分页,无列表, 直接id, articleid=xxx */
	private void handleNoListPages() {
		int start = site.getStartPage() + site.getSkipCount();
		int count = site.getPageCount() - site.getSkipCount();
		Page page = site.getContentPage();
		if (page == null)
			return;
		String url = page.getUrl();
		if (url == null || !url.contains("${PAGE_ID}")) {
			return;
		}
		for (int i = 0; i < count; i++) {
			int p = start + i;
			String id = String.valueOf(p);
			String u = StringUtils.replace(url, "${PAGE_ID}", id);
			if (!handleContentPage(null, u, id, null))
				break;
			if (count2 % 10 == 0)
				System.out.println(count1 + "/" + count2 + "/" + max);
			if (max > 0 && count2 >= max)
				break;
		}
	}
	
	private boolean handleContentPage(String referer, String url, String id, String title) {
		String path = null;
		if (title != null) {
			path = getPath(id, title);
			if (new File(path).exists()) {
				count2 ++;
				return true;
			}
		}
		url = getUrl(referer, url);
		System.out.println(url);
		Page page = site.getContentPage();
		GetMethod get = getGetMethod(url, referer, page.isFollowRedirects());
		try {
			String content = execute(client, get);
			if (content == null)
				return true;
			if (checkShield(content))
				return false;
			if (!successful(page, content))
				return true;
			content = cut(content, page.getStartString(), page.getEndString());
			if (title == null && page.getTitleRegex() != null) {
				Pattern p = Pattern.compile(page.getTitleRegex());
				Matcher m = p.matcher(content);
				if (m.find()) {
					title = m.group(1);
				}
			}
			if (StringUtils.isNotEmpty(page.getContentRegex())) {
				Pattern p = Pattern.compile(page.getContentRegex());
				Matcher m = p.matcher(content);
				if (m.find()) {
					int[] groups = page.getGroups();
					if (groups == null || groups.length == 0)
						groups = new int[] {1};
					String article = m.group(groups[0]);
					if (path == null) {
						if (title == null && groups.length > 1)
							title = m.group(groups[1]);
						if (title == null)
							title = id + "-" + (System.currentTimeMillis());
						path = getPath(id, title);
						if (new File(path).exists()) {
							count2 ++;
							return true;
						}
					}
					saveArticle(path, title, article);
					count1++;
					count2++;
				}
			} else {
				saveArticle(path, title, content);
				count1++;
				count2++;
			}
			sleep();
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			get.releaseConnection();
		}
		return true;
	}
	
	private void saveArticle(String path, String title, String content) {
		try {
			File f = new File(path);
			if (!f.getParentFile().exists())
				f.getParentFile().mkdirs();
			OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(path), "UTF-8");
			if (SAVE_TITLE)
				out.write(title + "\r\n");
			if (site.getContentHandler() != null)
				content = site.getContentHandler().handler(content);
			if (content == null)
				return;
			content = original(Tidy.html2text(content));
			if (content == null)
				return;
			out.write(content);
			out.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	String getPageUrl(String url, int page) {
		if (site.getPageKey() == null)
			return url;
		Map<String, String> params = url2map(url);
		switch (site.getPageMode()) {
		case ArticleSite.PAGE_MODE_NORMAL:
			params.put(site.getPageKey(), String.valueOf(page));
			return map2url(params);
		case ArticleSite.PAGE_MODE_SOAP:
			
			break;
		case ArticleSite.PAGE_MODE_RECORD:
			params.put(site.getPageKey(), String.valueOf(page * site.getPageSize()));
			return map2url(params);
		default:
			break;
		}
		return null;
	}
	
	private String getPath(String id, String title) {
		String tpl = site.getPathTemplate();
		//{BASE}\\{SITE}\\{QUERY}\\{MONTH}\\{DAY}\\{ID}-{TITLE}.txt
		tpl = replace(tpl, "{BASE}", SAVE_PATH);
		tpl = replace(tpl, "{SITE}", site.getName());
		tpl = replace(tpl, "{ID}", getSiteId(id));
		if (site.getTitleHandler() != null)
			title = site.getTitleHandler().handler(title);
		tpl = replace(tpl, "{TITLE}", nicename(title));
		tpl = replace(tpl, "{QUERY}", site.getQuery());
		tpl = replace(tpl, "{PAGE_SCOPE}", site.getStartPage() + "-" + (site.getStartPage() + site.getPageCount()));
		Calendar cal = Calendar.getInstance();
		cal.setTime(new Date());
		tpl = replace(tpl, "{YEAR}", String.valueOf(cal.get(Calendar.YEAR)));
		tpl = replace(tpl, "{MONTH}", String.valueOf(cal.get(Calendar.MONDAY)+1));
		tpl = replace(tpl, "{DAY}", String.valueOf(cal.get(Calendar.DAY_OF_MONTH)));
		return tpl;
	}
	
	private String replace(String tpl, String token, String replace) {
		if (tpl.contains(token))
			return StringUtils.replace(tpl, token, replace);
		return tpl;
	}
	
	private String getSiteId(String id) {
		if (StringUtils.isNotEmpty(id))
			return id;
		return String.format("No%03d", siteId++);
	}
	
	private String nicename(String name) {
//		System.out.println(name);
		name = Tidy.html2text(name.trim());
//		name = Tidy.replaceNumber(name);
//		System.out.println(name);
		name = Tidy.toFilename(name);
//		name = StringUtils.replaceChars(name, "'\"\\/<>?*\r\n\t:.$", "`--").trim();
		name = StringUtils.capitalize(name);
		return name;
	}
	
	private void sleep() {
		if (site.getInterval() > 0)
			try {
				Thread.sleep(site.getInterval());
			} catch (Exception e) {
			}
	}
	
	/** 检测是否被屏蔽 */
	private boolean checkShield(String resp) {
		if (resp == null || site.getShieldString() == null) {
			return shield;
		}
		if (resp.contains(site.getShieldString())) {
			shield = true;
			return true;
		}
		return shield;
	}
	
	protected String getCharset(HttpMethodBase method) {
		if (site.getCharset() != null)
			return site.getCharset();
		return method.getResponseCharSet();
	}
	
	/** 伪原创, 段落重排 */
/*
	public static String original2(String article) {
		int start = 2500, p = -1;
		while (p == -1 && start >= 10) {
			p = article.indexOf('.', start);
			if (start > 1000)
				start = start - 1000;
			else
				start = start / 10;
		}
		if (p == -1)
			return article;
		int p2 = article.indexOf('.', p + 10);
		if (p2 == -1) {
			return article.substring(p + 1) + "\r\n" + article.substring(p + 1);
		} else {
			return article.substring(p2 + 1) + "\r\n" + article.substring(p+1, p2+1) + "\r\n" + article.substring(0, p+1); 
		}
	}
/*/
	public static String original(String article) {
		article = article.trim();
		if (article.length() == 0)
			return null;
		article = Tidy.mergeBlanks(article);
		article = Tidy.removeUrls(article);
//		article = Tidy.replaceNumber(article);
		article = Tidy.removeBlankLines(article);
		article = Tidy.replaceChars(article);
		List<String> paragraphs = new ArrayList<String>();
		int n, p = 100, l = 0;
		while ((n = article.indexOf('.', p)) != -1) {
			paragraphs.add(article.substring(l, n + 1).trim());
			l = n + 1;
			p = n + 180 + Utils.random(400);
		}
		paragraphs.add(article.substring(l));
		n = paragraphs.size() * 2 / 3 + 1;
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < n; i++) {
			int j = Utils.random(n - i);
			String para = paragraphs.remove(j);
			sb.append(toParagraph(para));
		}
		for (String para : paragraphs) {
			sb.append(toParagraph(para));
		}
		return sb.toString();
	}
	
	/** 增加前导空格和首字母大写 */
	private static String toParagraph(String content) {
		return "\t" + StringUtils.capitalize(content) + "\r\n";
	}
//*/
	
	private String cut(String resp, String start, String end) {
		if (resp == null)
			return resp;
		if (StringUtils.isNotEmpty(start) && resp.contains(start))
			resp = resp.substring(resp.indexOf(start));
		if (StringUtils.isNotEmpty(end) && resp.contains(end))
			resp = resp.substring(0, resp.indexOf(end) + end.length());
		return resp;
	}
}
